# -*- coding: utf-8 -*-

import scrapy

import re

class terryBlogSpider(scrapy.Spider):
    name =  'terryBlog'
    allowed_domins  = ["terrytao.wordpress.com"]
    # allowed_domins  = ["https://terrytao.wordpress.com/career-advice/"]
    start_urls  = [
        "https://terrytao.wordpress.com/career-advice/",
        "https://terrytao.wordpress.com/advice-on-writing-papers/",
    ]

    def parse(self,response):
        
        contents = response.selector.css('.post-content')
        urlsTextCareer = contents.css('a[href*=career-advice]').extract()
        urlsTextWriting  = contents.css('a[href*=advice-on-writing-papers]').extract()

        urlsCareer = list()

        for urlText  in urlsTextCareer:
            url  = re.search('(?<=")https(.*)(?=")',urlText) # this is a unicode string
            urlsCareer.append(url)

        urlsWriting = list()

        for urlText  in urlsTextWriting:
            url  = re.search('(?<=")https(.*)(?=")',urlText) # this is a unicode string
            urlsWriting.append(url)

        urls = urlsCareer+  urlsWriting 
        # print(len(urls))

        #parseBlogContents
        blogItem = {};
        
        titles = response.selector.css('.post-title::text').extract()
        blogItem['title'] = titles[0]

        contents = response.selector.css('.post-content').extract()
        # see http://stackoverflow.com/questions/24358361/removing-u2018-and-u2019-character  
        blogItem['content']  = contents[0]
        
        #blogItem type
        ad   =  re.search('career-advice',response.url)
        if(ad == None):
            blogItem['type']='advice-on-writing-papers'
        else:
            blogItem['type']='career-advice'

        blogItem['depth'] = response.meta['depth']


        # yield scrapy.Request(urls, callback=self.parse)
        for url in urls:
            url1  = url.group(0)
            if(url1.find('?')== -1 and url1.find('#')==-1 ):
                # print url1
                yield scrapy.Request(url1, callback=self.parse)
        
        yield blogItem
